package com.atguigu.sparkcore.day01.singlevalue

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * Author atguigu
 * Date 2020/10/27 16:16
 */
object CoaleaseDemo {
    def main(args: Array[String]): Unit = {
        val conf: SparkConf = new SparkConf().setAppName("CoaleaseDemo").setMaster("local[2]")
        val sc: SparkContext = new SparkContext(conf)
        val list1 = List(3, 5, 70, 60, 10, 20)
        val rdd1: RDD[Int] = sc.parallelize(list1, 3)
        
        //        val rdd2 = rdd1.coalesce(4)
        //                val rdd3: RDD[Array[Int]] = rdd2.glom()
        val rdd2 = rdd1.repartition(2)
        val num: Int = rdd2.getNumPartitions
        println(num)
        rdd2.glom().collect.foreach(x => println(x.mkString(",")))
        sc.stop()
        
    }
}

/*
coalesce
    默认情况不shuffle, 所以只能减少不能增加
repartition
    即可以增加分区, 也可以减少分区, 因为会shuffle
    
    repartition 之后, 一定不会数据倾斜
    
 选择:
    如果减少分区, 尽量避免shuffle, 尽量选择coalesce
    如果增加分区, 只能选择repartition
    
 */
